{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 1. 爬虫过程:\n",
    "URL ->发送请求 -> 获取数据 -> 保存数据\n",
    "\n",
    "### 2. 爬虫系统\n",
    "URL管理器\n",
    "数据下载器\n",
    "数据分析器\n",
    "数据保存器\n",
    "调度器\n",
    "\n",
    "### 3. Request\n",
    "请求url:\n",
    "请求方式: POST, GET\n",
    "请求头: User-Agent, Cokie\n",
    "请求体: 登录账户, 密码\n",
    "\n",
    "### 4. 相应状态\n",
    "    2** 成功\n",
    "    3** 重定向\n",
    "    4** 客户端错误\n",
    "    5** 服务器错误\n",
    "\n",
    "### HTTP过程\n",
    "连接 -> 请求 -> 相应 -> 断开连接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# urllib\n",
    "# 发送请求\n",
    "import urllib\n",
    "\n",
    "bing = \"https://bing.com\"\n",
    "r = urllib.request.urlopen(bing)\n",
    "print(r.getcode())\n",
    "print(r.geturl())\n",
    "# print(r.getheader())\n",
    "print(r.getheaders())\n",
    "\n",
    "# print(r.getheaders())\n",
    "print(r.read())\n",
    "\n",
    "bing = \"https://bing.com/search\"\n",
    "data={'q':'python'}\n",
    "req_data=urllib.parse.urlencode(data)\n",
    "req_url=bing+'?'+req_data\n",
    "r=urllib.request.urlopen(req_url)\n",
    "print(r.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 获取Cookie 不通\n",
    "import http.cookiejar, urllib.request\n",
    "\n",
    "bing = \"https://bing.com\"\n",
    "\n",
    "cookie = http.cookiejar.Cookiejar()\n",
    "handler = urllib.request.HTTPCookieProcessor(cookie)\n",
    "opener = urllib.request.bulid_opener(handler)\n",
    "response = opener.open(bing)\n",
    "for i in cookie:\n",
    "    print(i.name + \" = \" + i.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "bing = \"https://bing.com\"\n",
    "\n",
    "r = requests.get(bing)\n",
    "print(r.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mRunning cells with 'Python 3.10.8 64-bit' requires ipykernel package.\n",
      "\u001b[1;31mRun the following command to install 'ipykernel' into the Python environment. \n",
      "\u001b[1;31mCommand: '/bin/python -m pip install ipykernel -U --user --force-reinstall'"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "dirver=webdriver.Edge\n",
    "dirver.get(url='http://www.bing.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理header\n",
    "# GET /images/lol/act/img/js/heroList/hero_list.js?ts=2783292 HTTP/2\n",
    "h_txt=\"\"\"\n",
    "Host: game.gtimg.cn\n",
    "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0\n",
    "Accept: */*\n",
    "Accept-Language: en-US,en;q=0.5\n",
    "Accept-Encoding: gzip, deflate, br\n",
    "Origin: https://101.qq.com\n",
    "Connection: keep-alive\n",
    "Referer: https://101.qq.com/\n",
    "Sec-Fetch-Dest: empty\n",
    "Sec-Fetch-Mode: cors\n",
    "Sec-Fetch-Site: cross-site\n",
    "Pragma: no-cache\n",
    "Cache-Control: no-cache\n",
    "TE: trailers\n",
    "\"\"\"\n",
    "header={}\n",
    "s1=h_txt.split('\\n')\n",
    "for i in s1:\n",
    "  s2=i.split(': ')\n",
    "  if(len(s2)==1):\n",
    "    continue\n",
    "  header[s2[0]]=s2[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Host': 'game.gtimg.cn',\n",
       " 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0',\n",
       " 'Accept': '*/*',\n",
       " 'Accept-Language': 'en-US,en;q=0.5',\n",
       " 'Accept-Encoding': 'gzip, deflate, br',\n",
       " 'Origin': 'https://101.qq.com',\n",
       " 'Connection': 'keep-alive',\n",
       " 'Referer': 'https://101.qq.com/',\n",
       " 'Sec-Fetch-Dest': 'empty',\n",
       " 'Sec-Fetch-Mode': 'cors',\n",
       " 'Sec-Fetch-Site': 'cross-site',\n",
       " 'Pragma': 'no-cache',\n",
       " 'Cache-Control': 'no-cache',\n",
       " 'TE': 'trailers'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js?ts=2783292'\n",
    "\n",
    "resp=requests.get(url=url)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_85622/2391791917.py:3: DeprecationWarning: invalid escape sequence '\\/'\n",
      "  json_str=resp.content.decode('unicode_escape')\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "json_str=resp.content.decode('unicode_escape')\n",
    "# json_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "with open(\"../lol_hero.json\",'w') as f:\n",
    "  f.write(json_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "https://101.qq.com/#/hero-detail?heroid=1\n",
    "\n",
    "https://game.gtimg.cn/images/lol/act/img/skinloading/1000.jpg\n",
    "\n",
    "https://game.gtimg.cn/images/lol/act/img/skinloading/2000.jpg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 爬取西瓜图片\n",
    "import requests \n",
    "url=r'https://cn.bing.com/images/search?q=%e8%a5%bf%e7%93%9c+&form=HDRSC2&first=1'\n",
    "resp=requests.get(url)\n",
    "resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resp.content.decode('unicode_escape')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_84676/3023681573.py:2: DeprecationWarning: invalid escape sequence '\\/'\n",
      "  html = etree.HTML(resp.content.decode('unicode_escape'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Element html at 0x7fc9d411fbc0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lxml import etree\n",
    "html = etree.HTML(resp.content.decode('unicode_escape'))\n",
    "html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '/rp/N7tKdYmlODF3joFiKWv51yR3FE4.svg',\n",
       " '/rp/kFAqShRrnkQMbH6NYLBYoJ3lq9s.png',\n",
       " 'https://tse2-mm.cn.bing.net/th/id/OIP-C.y1S8EA3ej8toWq8R3hetfgHaG6?w=201&h=188&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse4-mm.cn.bing.net/th/id/OIP-C.OovkfA3i-TNSNaOQO4uA2QHaE8?w=281&h=188&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse2-mm.cn.bing.net/th/id/OIP-C.PPQrPo3-NAb18dhjy8fY9wHaFl?w=248&h=188&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse3-mm.cn.bing.net/th/id/OIP-C.wizZALZ8eEbWC1JQ_2nVXAHaFj?w=250&h=188&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse1-mm.cn.bing.net/th/id/OIP-C.vC3O5-ai9u73vG7s8ajGuQHaE8?w=280&h=187&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse2-mm.cn.bing.net/th/id/OIP-C.BrhKfwAIutP_jD5TtWMZQAHaF7?w=234&h=187&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse1-mm.cn.bing.net/th/id/OIP-C.43Ks3_s6oIarYVUolB1LpwHaE8?w=280&h=187&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse4-mm.cn.bing.net/th/id/OIP-C.sKkp-o3qLtlo3HcCJ9lBFwHaHe?w=186&h=187&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse1-mm.cn.bing.net/th/id/OIP-C.M3LQfx5glSSuz5-vMyXxHQHaE7?w=249&h=180&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse1-mm.cn.bing.net/th/id/OIP-C.ssJq-I54hBUVRB4oNHjX5gAAAA?w=236&h=180&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse4-mm.cn.bing.net/th/id/OIP-C.95bOeRtIqfg0AT2ecvDdvAAAAA?w=247&h=180&c=7&r=0&o=5&pid=1.7',\n",
       " 'https://tse2-mm.cn.bing.net/th/id/OIP-C.UbCVZQ8SU77PRUpM1-W47wHaE7?w=248&h=180&c=7&r=0&o=5&pid=1.7',\n",
       " '/sa/simg/Flag_Feedback.png',\n",
       " '/rp/ytiieusXgM2K8bLkEDP-AS1ePds.png']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xpath=\"//img/@src\"\n",
    "nodes=html.xpath(xpath)\n",
    "nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=0\n",
    "for i in nodes:\n",
    "    if not i.startswith('http'):\n",
    "        continue\n",
    "    resp=requests.get(i)\n",
    "    data=resp.content\n",
    "    \n",
    "    surrfix=i[i.rfind('.'):-1]\n",
    "    with open(f'./img/{x}.jpg','wb') as f:\n",
    "        x+=1\n",
    "        f.write(data)\n",
    "      "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4c2b8563932708ac4f64e3bc4797ea8eb43fc7530baa7f01daadf5b46d7b1fd4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
